/* Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
  <file> ulf_gen.c </file>
  <brief>
    Generating ULF(Unified Lexical Frequencies) file gathering statistics of keywords usage.
    This file is part of ULS, Unified Lexical Scheme.
  </brief>
  <author>
    Stanley J. Hong <babel96@gmail.com>, August 2011.
  </author>
*/

#include "common.h"
#include "uls.h"

#ifdef WIN32
#define THIS_PROGNAME "UlfGen";
#else
#define THIS_PROGNAME "ulf_gen";
#endif

#define DFL_N_ITERS 0xFFFF

typedef struct _stat_of_round stat_of_round_t;
struct _stat_of_round  {
	int        n_hcodes;
	uls_uint32 hcodes[ULF_MAXDIM_HASHSPACE];
	int   n; // # of buckets
	double avg;
	double sigma2;
	double gamma;
};

char *progname;
char *config_file;
char *target_dir;
char home_dir[1024];
char *filelist;
FILE *fp_list;
char *out_dir;
int  opt_verbose;

uls_lex_t *sam_lex;

uls_kwrd_stat_t *kwrd_stat_list;
int n_kwrd_stat_list;

int target_hashtable_size;
int dim_of_hspace;
int n_iters;

#ifdef HAVE_GETOPT_H
#include <getopt.h>

static const struct option longopts[] = {
	{ "lang",  required_argument,      NULL, 'L' },
	{ "list",  required_argument,      NULL, 'l' },
	{ "dest-dir",  required_argument,  NULL, 'd' },
	{ "num-iter",  required_argument,  NULL, 'n' },
	{ "size-hash",  required_argument, NULL, 's' },
	{ "verbose",  no_argument,         NULL, 'v' },
	{ "version", no_argument,          NULL, 'V' },
	{ "help",    no_argument,          NULL, 'h' },
	{ NULL, 0, NULL, 0 }
};
#endif

static void usage(void)
{
	printf("Usage: %s -L <ulc-file> [OPTIONS] <file|dir>\n\n", progname);
	printf("  or:  %s -L <ulc-file> <file1> <file2> ...\n\n", progname);
	printf("  or:  %s -L <ulc-file> -d /usr/local/share <file1> <file2> ...\n\n", progname);
	printf("  or:  %s -L <ulc-file> -l <list-file> <target-dir>\n\n", progname);

	printf("%s collects the usage statistics of given language specified", progname);
	printf(" by sampling from source code files.\n");
	printf("%s outputs the frequencies of keywords and stores them in a file suffixed by '*.ulf'.\n", progname);
	printf("The output file of 'sample.ulc' is 'sample.ulf'.");
	printf(" and should be in the same directory of ulc-file(*.ulc) in order to work correctly.\n");
	printf("To save the output-file to other one than default(==current-direcotry), use -d-option.\n");

	printf("  -L, --lang=<ulc-spec>   specify the lexical-spec(*.ulc) of the language.\n");
	printf("  -l, --list=<list-file>  specify the list of data source files.\n");
	printf("  -d, --dest-dir=<a-dir>  specify the target directory of the output file(*.ulf).\n");
	printf("  -v, --verbose           verbose mode.\n");
	printf("  -V, --version           output version information and exit.\n");
	printf("  -h, --help              display this help and exit.\n\n");
	printf("\n");

	printf("To get the ulf-file(*.ulf),\n");
	printf("    %s -L sample.ulc a.c b.c src/d.c\n", progname);
	printf("\n");

	printf("To specify the target directory of the resultant file(*.ulf),\n");
	printf("    %s -d /opt/share -L sample src/main.c src/main.h\n", progname);
	printf("\n");

	printf("To process all the filepath in 'a.list' in '/package/home'\n");
	printf("    %s -L sample -l a.list /package/home\n", progname);
	printf("\n");
}

static int
ulfgen_options(int opt, char* optarg)
{
	int   stat = 1;
	int   siz;

	switch (opt) {
	case 'L':
		config_file = optarg;
		break;

	case 'l':
		filelist = optarg;
		break;

	case 'n':
		n_iters = atoi(optarg);
		if (n_iters <= 0) n_iters = 1;
		break;

	case 'd':
		out_dir = optarg;
		break;

	case 's':
		if ((siz=atoi(optarg)) > 0) {
			target_hashtable_size =  siz;
		} else {
			err_panic("non-positive hash-size(%d) specified!", siz);
		}
		break;

	case 'v':
		++opt_verbose;
		break;

	case 'V':
		printf("%s %s, written by %s,\n\tis provided under %s.\n",
			progname, ULF_GEN_PROGVER, ULS_AUTHOR, ULS_LICENSE_NAME);
		stat = 0;
		break;

	case 'h':
		usage();
		stat = 0;
		break;

	default:
		err_log("undefined option -%c", opt);
		stat = -1;
		break;
	}

	return stat;
}

static int
parse_options(int argc, char* argv[])
{
#ifdef HAVE_GETOPT_H
	int   rc, opt, longindex;
#endif
	int i0;

	progname = THIS_PROGNAME;
	target_hashtable_size = ULF_HASH_TABLE_SIZE;
	dim_of_hspace = 2;

	filelist = NULL;
	out_dir = ".";
	target_dir = NULL;
	n_iters = DFL_N_ITERS;

#ifdef HAVE_GETOPT_H
	while ((opt=getopt_long(argc, argv, "L:l:d:n:s:vVh", longopts, &longindex)) != -1) {
		if ((rc=ulfgen_options(opt, optarg)) <= 0) {
			return rc;
		}
	}
	i0 = optind;
#else
	if ((i0=uls_getopts(argc, argv, "L:l:d:n:s:vVh", ulfgen_options)) <= 0) {
		return i0;
	}
#endif

	if (filelist != NULL) {
		if (i0 >= argc) {
			err_log("Specify the target-directory to apply the paths in %s", filelist);
			return -1;
		}

		if ((fp_list=fopen(filelist, "r"))==NULL) {
			err_log("%s: fail to read '%s'", __FUNCTION__, filelist);
			return -1;
		}

		target_dir = argv[i0++];
	}

	return i0;
}

int
incl_if_keyw(const char* kwrd)
{
	uls_kwrd_stat_t *kwrd_stat;
	int stat = 0;

	kwrd_stat = uls_search_kwrd_stat_sorted(
		kwrd_stat_list, n_kwrd_stat_list, kwrd);

	if (kwrd_stat != NULL) {
		if (kwrd_stat->freq < INT_MAX) {
			++kwrd_stat->freq;
			stat = 1;
		} else {
			stat = -1;
		}
	}

	return stat;
}

int
proc_file(char *fpath)
{
	int tok;

	if (fpath == NULL) return 0;

	uls_push_file(sam_lex, fpath, 0);
	if (opt_verbose >= 1)
		err_log("Processing %s, ...", fpath);

	for ( ; ; ) {
		tok = uls_gettok(sam_lex);
		if (uls_is_eoi(sam_lex)) break;

		incl_if_keyw(uls_lexeme_text(sam_lex));
	}

	return 1;
}

int
proc_filelist(FILE *fin)
{
	char  linebuff[1024], *lptr, *fpath;
	int   i, len, lno = 0, stat=0;

	while (fgets(linebuff, sizeof(linebuff), fin) != NULL) {
		++lno;

		if ((len=strlen(linebuff))>0 && linebuff[len-1]=='\n') {
			linebuff[--len] = '\0';
		}

		for (lptr=linebuff; *lptr==' ' || *lptr=='\t'; lptr++)
			/* NOTHING */;

		if (*lptr=='\0' || *lptr=='#') continue;

		fpath = lptr;
		for (i=strlen(fpath)-1; i>=0; i--) {
			if (!isspace(fpath[i])) break;
		}
		fpath[++i] = '\0';

		if (proc_file(fpath) < 0) {
			err_log("fail to process '%s'", fpath);
			stat = -1;
			break;
		}
	}

	return stat;
}

void
go_round(int n_hcodes, uls_uint32* hcodes, int* hash_range, int n_hash_range, stat_of_round_t* p_stat)
{
	uls_kwrd_stat_t *e;
	int i, hash_id, n;
	double sum, sum1, sum2;
	
	memset(hash_range, 0x00, n_hash_range * sizeof(int));

	memcpy(p_stat->hcodes, hcodes, n_hcodes * sizeof(uls_uint32));
	p_stat->n_hcodes = n_hcodes;

	sum = 0.;
	for (i=0; i < n_kwrd_stat_list; i++) {
		e = kwrd_stat_list + i;

		hash_id = ulf_hashfunc(e->kwrd, n_hash_range, -1, n_hcodes, hcodes); 

		if (hash_range[hash_id] < INT_MAX)
			++hash_range[hash_id];

		sum += hash_range[hash_id] * e->freq;
	}

	sum1 = sum2 = 0.;
	n = 0;
	for (hash_id=0; hash_id < n_hash_range; hash_id++) {
		if ((i=hash_range[hash_id]) > 0) {
			sum1 += i;
			sum2 += i*i;
			++n;
		}
	}

	p_stat->avg = sum1 / n;
	p_stat->sigma2 = sum2 / n - p_stat->avg * p_stat->avg;
	p_stat->n = n;
	p_stat->gamma = sum / n;
}

void
get_random_hcodes(int n_hcodes0, uls_uint32* hcodes0)
{
	uls_uint32 hcode;
	int i, j;

	for (i=0; i<n_hcodes0; i++) {
		hcode = 0;
		for (j=0; j<4; j++) {
			hcode ^= rand();
			hcode <<= 8;
		}
		hcodes0[i] = hcode;
	}
}

void
calc_good_hcode0(int n_samples, int n_hcode0, uls_uint32* hcode0)
{
	int *bucket_size_array;
	stat_of_round_t  best_round_stat, round_stat;
	int i;

	bucket_size_array = (int *) malloc(target_hashtable_size * sizeof(int));
	if (bucket_size_array == NULL) {
		err_log("can't malloc bucket_size_array[]");
		return;
	}

	for (i=0; i<n_samples; i++) {
		get_random_hcodes(n_hcode0, hcode0);
		go_round(n_hcode0, hcode0, bucket_size_array, target_hashtable_size, &round_stat);

		if (opt_verbose >= 2 && i % 1000 == 0) {
			uls_printf(" avg = %.3f, sigma2 = %.3f, n=%d, gamma=%.3f\n", 
				round_stat.avg, round_stat.sigma2, round_stat.n, round_stat.gamma);
		}

		if (i > 0) {
			if (round_stat.gamma < best_round_stat.gamma) {
				best_round_stat = round_stat;
			}
		} else {
			best_round_stat = round_stat;
		}
	}

	if (opt_verbose >= 1) {
		uls_printf("*** avg = %.3f, sigma2 = %.3f, n=%d, gamma=%.3f\n", 
			best_round_stat.avg, best_round_stat.sigma2, best_round_stat.n, best_round_stat.gamma);
	}

	memcpy(hcode0, best_round_stat.hcodes, n_hcode0 * sizeof(uls_uint32));

	if (opt_verbose >= 2) {
		uls_printf(" hcode[] = ");
		for (i=0; i<best_round_stat.n_hcodes; i++) {
			uls_printf(" 0x%08X", best_round_stat.hcodes[i]);
		}
		uls_printf("\n");
	}

	free(bucket_size_array);
}

static int kwrd_stat_comp_by_freq_reverse(const void* a, const void* b)
{
	uls_kwrd_stat_t *a1 = (uls_kwrd_stat_t *) a;
	uls_kwrd_stat_t *b1 = (uls_kwrd_stat_t *) b;

	return b1->freq - a1->freq;
}

int
main(int argc, char* argv[])
{
	FILE *fp_out;
	char out_file[1024];
	char *conf_fname, *ptr;
	int i0, i, conf_fname_len, stat=0;
	uls_uint32 hcode0[ULF_MAXDIM_HASHSPACE];
#ifdef WIN32DEBUG
	initialize_uls();
#endif
	uls_getcwd(home_dir, sizeof(home_dir));

	if ((i0=parse_options(argc, argv)) <= 0) {
		if (i0 < 0) err_log("Incorrect use of command options.");
		return i0;
	}

	if (fp_list == NULL && i0 >= argc) {
		err_log("%s(%s), written by %s,\n\tis provided under %s.",
			progname, ULF_GEN_PROGVER, ULS_AUTHOR, ULS_LICENSE_NAME);
		err_log("Type '%s -h' for usage.", progname);
		return 0;
	}

	if (config_file == NULL) {
		err_log("Specify ulc-file(*.ulc) with -L-option");
		return -1;
	}

	/* out_file: *.ulf */
	conf_fname = uls_filename(config_file);
	if ((ptr=strrchr(conf_fname, '.'))!=NULL) {
		conf_fname_len = (int) (ptr - conf_fname);
	} else {
		conf_fname_len = strlen(conf_fname);
	}

	strcpy(out_file, out_dir);
	i = strlen(out_file);
	out_file[i++] = '/';
	memcpy(out_file+i, conf_fname, conf_fname_len);
	i += conf_fname_len;
	strcpy(out_file+i, ".ulf");

	if (opt_verbose)
		uls_list_ulc_searchpaths(config_file);

	if ((sam_lex=uls_create(config_file)) == NULL) {
		err_log("can't create a uls-object for lexical analysis.");
		return -1;
	}

	if ((kwrd_stat_list=uls_export_kwtable(sam_lex, &n_kwrd_stat_list)) == NULL) {
		err_log("No keyword infomation!");
		return -1;
	}

	// kwrd_stat_list is sorted by kwrd
	for (i=0; i < n_kwrd_stat_list; i++) {
		kwrd_stat_list[i].freq = 0;
	}

	err_log("Gathering the statistics of keywords usage, ...");

	if (fp_list != NULL) {
		if (uls_chdir(target_dir) < 0) {
			err_log("can't change to %s", target_dir);
			uls_free_kwrd_stat_list(kwrd_stat_list);
			uls_destroy(sam_lex);
			return -1;
		}

		proc_filelist(fp_list);
		fclose(fp_list);
		uls_chdir(home_dir);

	} else {
		for (i=i0; i<argc; i++) {
			proc_file(argv[i]);
		}
	}

	qsort(kwrd_stat_list, n_kwrd_stat_list, sizeof(uls_kwrd_stat_t),
		kwrd_stat_comp_by_freq_reverse);

	calc_good_hcode0(n_iters, dim_of_hspace, hcode0);

	err_log("Writing the frequencies of keywords to %s, ...", out_file);
	if ((fp_out=fopen(out_file, "w")) == NULL) {
		err_log("%s: fail to create '%s'", __FUNCTION__, out_file);
		stat = -1;
	} else {
		ulf_create_file(dim_of_hspace, hcode0, target_hashtable_size, kwrd_stat_list, n_kwrd_stat_list, fp_out);
		fclose(fp_out);
	}

	uls_free_kwrd_stat_list(kwrd_stat_list);
	uls_destroy(sam_lex);

	return stat;
}
